# -*- coding: UTF-8 -*-
# Author: Damon(wuud1)
# CreateDate: 
# Message:

from MysqlTools import MysqlTools
from config import *
import requests
import os
import json
import hashlib
import time, re

class HK_Spider:
    def __init__(self, **kwargs):
        self.db = MysqlTools(db_config=kwargs.get("db_config"), type='dict', auto_commit=1000)
        self.brands = kwargs.get("brands")
        self.regn = kwargs.get("regn")
        self.html_path = os.getcwd() + os.sep + '{}_html/'.format(self.brands)
        self.img_path = os.getcwd() + os.sep + '{}_img/'.format(self.brands)
        if not os.path.exists(self.html_path):
            os.makedirs(self.html_path)
        if not os.path.exists(self.img_path):
            os.makedirs(self.img_path)
        self.cookies = COOKIES
        self.headers = HEADERS
        self.post_data = POST_DATA
        self.hkgb = BRAND_DICT[self.brands]['hkgb']

    def get_md5(self, str):
        '''获取md5加密电子指纹'''
        md5 = hashlib.md5()
        md5.update(str)
        return md5.hexdigest()

    def parse_url(self, url, method, post_data=None, headers=None, tofile=True, timeout=30, **kwargs):
        requests.packages.urllib3.disable_warnings()  # 因为把ssl安全验证关了,所以不让代码报不安全的https请求警告
        key = kwargs.get("key", "")
        proxies = kwargs.get("proxies")
        charset = kwargs.get("charset", "utf8")  # 暂时没有用到
        allow_redirects = kwargs.get("allow_redirects", False)  # 是否允许重定向, 默认不重定向,获取原始响应
        if method == 'GET':
            data = "GET" + url + key  # 拼接data
            md5 = self.get_md5(data.encode())  # 获取此次请求电子指纹
            file_path = self.html_path + md5 + ".html"
            if os.path.exists(file_path):  # 判断文件是否存在,如果存在就从本地读取
                print("GET for local")
                with open(file_path, 'r') as f:
                    res = f.read()
                response = res
            else:  # 如果路径不存在, 从web端请求
                print("GET for web")
                try:
                    response = requests.get(url=url, verify=False, headers=headers, cookies=self.cookies, timeout=timeout,
                                            allow_redirects=allow_redirects, proxies=proxies)
                except Exception as e:
                    response = requests.get(url=url, verify=False, headers=headers, cookies=self.cookies, timeout=timeout,
                                            allow_redirects=allow_redirects, proxies=proxies)

                # 每次请求都获取cookies, 并更新self.cookies(如果获取到cookies的话)
                cookies = response.cookies.get_dict()
                for k in cookies:
                    self.cookies[k] = cookies[k]

                # 获取location,
                location = response.headers.get("Location", None)
                if location:
                    print("响应头中存在location")
                    self.location = location
                    print(location)
                    # return response.content.decode('utf8')

                if response:
                    if tofile:
                        with open(file_path, 'w') as f:  # 保存此次请求的html
                            f.write(response.content.decode('utf8'))
                    response = response.content.decode("utf8")
                else:
                    response = None

        elif method == "POST":
            data = "POST" + url + json.dumps(post_data)
            md5 = self.get_md5(data.encode())
            file_path = self.html_path + md5 + ".html"
            if os.path.exists(file_path):  # 判断文件是否存在,如果存在就从本地读取
                print("POST for local")
                with open(file_path, 'r') as f:
                    res = f.read()
                response = res
            else:
                print("POST for web")
                try:
                    response = requests.post(url=url, data=post_data, verify=False, headers=headers,
                                             cookies=self.cookies, timeout=timeout,
                                             allow_redirects=allow_redirects, proxies=proxies)
                except Exception as e:
                    response = requests.post(url=url, data=post_data, verify=False, headers=headers,
                                             cookies=self.cookies, timeout=timeout,
                                             allow_redirects=allow_redirects, proxies=proxies)

                # 获取cookies, 并更新self.cookies
                cookies = response.cookies.get_dict()
                for k in cookies:
                    self.cookies[k] = cookies[k]

                # 获取location,
                location = response.headers.get("Location", None)
                if location:
                    print("响应头中存在location")
                    self.location = location
                    print(location)
                    # response, file_path = self.parse_url(url=location, method="GET", headers=headers, local_pid=key)
                    # print "已重新请求location地址"
                    # return response, file_path
                    # return response.content.decode('utf8')

                if response:
                    if tofile:
                        with open(file_path, 'w') as f:  # 保存此次请求的html
                            f.write(response.content.decode('utf8'))
                    response = response.content.decode("utf8")
                else:
                    response = None
        return response, file_path

    def parse_one(self, response):
        '''解析kia或者hyundai大品牌下的车系'''
        res_l = response.split("\@")
        _l = []
        for info in res_l:
            if info == '':
                continue
            info_l = info.split("\:")
            # print(info_l)
            _dict = dict(
                brand=info_l[0],  # 品牌: KIA, HYUNDAI
                type=info_l[1],  # 类型: 乘客,SUV,ECO,商用的
                crnm=info_l[2],  # K3,K4,K5...
                type_code=info_l[4]  # 类型代码: 乘客:P, ECO:E
            )
            _l.append(_dict)
        return _l

    def get_cars_model_list(self):
        # 获取品牌下所有车系
        url = 'http://wpc.mobis.co.kr/Model?cmd=getModelBrandCarnameList&hkgb=A&hkmc={}&type=&views=Y&sgmts=C%2CE%2CP%2CR%2CS&bpnos=&regns={}%2CGEN&crnm='.format(self.hkgb, self.regn)
        try:
            response, file_path = self.parse_url(url=url, method='GET', headers=self.headers, proxies=PROXIES)
        except:
            response, file_path = self.parse_url(url=url, method='GET', headers=self.headers, proxies=PROXIES)
        if response:
            cars_model_list = self.parse_one(response)
        else:
            cars_model_list = []
        # return cars_model_list
        return [{'crnm': 'LAFESTA', 'type': '乘客', 'type_code': 'P'}]

    def parse_two(self, response):
        res_l = response.split("\@")
        _l = []
        for info in res_l:
            if info == '':
                continue
            info_l = info.split("\:")
            # print(info_l)
            _dict = dict(
                country=info_l[0],
                years=info_l[1],
                cid=info_l[4],
                name=info_l[5]
            )
            _l.append(_dict)
        return _l

    def get_cars_version_list(self, crnm, type_code):
        # 获取车系下所有版本
        url = 'http://wpc.mobis.co.kr/Model?cmd=getCatalogList2&hkgb={}&crnm={}&type={}&views=Y&sgmts=C%2CE%2CP%2CR%2CS&bpnos=&regns={}%2CGEN'.format(self.hkgb, crnm, type_code, self.regn)
        try:
            response, file_path = self.parse_url(url=url, method='GET', headers=self.headers, proxies=PROXIES)
        except:
            response, file_path = self.parse_url(url=url, method='GET', headers=self.headers, proxies=PROXIES)
        if response:
            try:
                cars_version_list = self.parse_two(response)
            except Exception as e:
                print(file_path)
                raise e
        else:
            cars_version_list = []
        return cars_version_list

    def parse_three(self, response):
        res_l = response.split('\@')
        _l = []
        for info in res_l:
            if info == '':
                continue
            info_l = info.split("\:")
            # print(info_l)
            _dict = dict(
                CRNM=info_l[0],  # name, 数据库limit字段
                BPNO=info_l[1],  # cid
                HKGB=info_l[2],  # 品牌代号 kia: K, hyundai: H
                HSGB=info_l[3],
                OPTY=info_l[4],
                PCCD=info_l[5],
                PKGB=info_l[6],
                VHCD=info_l[7],
                VIEW=info_l[8],
                EFIN=info_l[9],
                EFOT=info_l[10],
                GRTY=info_l[11],
                SGMT=info_l[12],
                UCCT=info_l[13]
            )
            _l.append(_dict)
        return _l[0]

    def get_cars_detail(self, cid):
        # 点击某一版本, 获取此版本详细信息, 用于后续请求
        url = 'http://wpc.mobis.co.kr/Model?cmd=getCatalogInfo2&bpno={}'.format(cid)
        try:
            response, file_path = self.parse_url(url=url, method='GET', headers=self.headers, proxies=PROXIES)
        except:
            response, file_path = self.parse_url(url=url, method='GET', headers=self.headers, proxies=PROXIES)
        if response:
            try:
                cars_detail = self.parse_three(response)
            except Exception as e:
                print(file_path)
                raise e
        else:
            cars_detail = {}
        return cars_detail

    def reset_post_data(self, cars_detail):
        # 重置post_data 基础参数
        self.post_data['CRNM'] = cars_detail.get("CRNM", "")
        self.post_data['BPNO'] = cars_detail.get("BPNO", "")
        self.post_data['HKGB'] = cars_detail.get("HKGB", "")
        self.post_data['HSGB'] = cars_detail.get("HSGB", "")
        self.post_data['PKGB'] = cars_detail.get("PKGB", "")
        self.post_data['OPTY'] = cars_detail.get("OPTY", "")
        self.post_data['PCCD'] = cars_detail.get("PCCD", "")
        self.post_data['VIEW'] = cars_detail.get("VIEW", "")
        self.post_data['EFIN'] = cars_detail.get("EFIN", "")
        self.post_data['EFOT'] = cars_detail.get("EFOT", "")
        self.post_data['VHCD'] = cars_detail.get("VHCD", "")
        self.post_data['GRTY'] = cars_detail.get("GRTY", "")
        self.post_data['SGMT'] = cars_detail.get("SGMT", "")
        self.post_data['UCCT'] = cars_detail.get("UCCT", "")
        self.post_data['ISVIN'] = "MODEL"
        self.post_data['AS'] = "N"
        self.post_data['MATCH'] = "ON"
        self.post_data['LANG'] = "CH"

    def parse_group_info(self, response):
        response = response.replace('@^', '')
        res_l = response.split("@")
        _l = []
        for info in res_l:
            # print(info)
            info_l = info.split(',')
            _dict = dict(
                code=info_l[0],
                group_name=info_l[1],
            )
            _l.append(_dict)
        return _l

    def get_cars_group_list(self, BPNO, PCCD, cid):
        url = 'http://wpc.mobis.co.kr/IllustSearch?cmd=getLargeGroupKeyList&bpno={}&pccd={}&pcno={}&vgtag=&vin='.format(BPNO, PCCD, cid)
        try:
            response, file_path = self.parse_url(url=url, method='GET', headers=self.headers, proxies=PROXIES)
        except:
            response, file_path = self.parse_url(url=url, method='GET', headers=self.headers, proxies=PROXIES)
        if response:
            try:
                cars_group_list = self.parse_group_info(response)
            except Exception as e:
                print(file_path)
                raise e
        else:
            cars_group_list = []
        return cars_group_list

    def parse_structrue_info(self, response):
        # 解析主组下,分组的信息
        res_0 = response.split("\@")[0].split('\^')
        res_1 = response.split("\@")[1].split('\^')
        _l0 = []
        _l1 = []
        for info in res_0:
            if info == '':
                continue
            info_l = info.split('\:')
            # print(info_l)
            _dict = dict(
                img_num=info_l[0],  # 描点在第几张图片上, 暂时没有意义
                mid=info_l[1],
                mid_imgPoint=[info_l[2], info_l[3], info_l[4], info_l[5]],
                mid_name=info_l[6]
            )
            _l0.append(_dict)
        for info1 in res_1:
            if info1 == '':
                continue
            info1_l = info1.split('\:')
            # print(info1_l)
            _dict1 = dict(
                mid=info1_l[0],  # 暂时不用此参数作为mid
                structrue_name=info1_l[1],
                GRNO=info1_l[2],  # 请求分组下数据用的code, 会跟mid不一样, 用此参数当做mid
                i3=info1_l[3],
                i4=info1_l[4],
                remark=info1_l[5],  # 分组备注, 此参数有内容的不多
                i6=info1_l[6],
            )
            _l1.append(_dict1)
        return _l0, _l1

    def get_cars_structrue_list(self, group_code):
        # 获取分组列表
        # POST请求
        url = 'http://wpc.mobis.co.kr/IllustSearch?cmd=getUniqueGroupKeyList'
        self.post_data['LRCD'] = group_code  # 添加主组号到post_data中
        try:
            response, file_path = self.parse_url(url=url, method='POST', headers=self.headers, post_data=self.post_data, proxies=PROXIES)
        except:
            response, file_path = self.parse_url(url=url, method='POST', headers=self.headers, post_data=self.post_data, proxies=PROXIES)

        if response:
            try:
                l1, cars_structrue_list = self.parse_structrue_info(response)  # l1 暂时无用
            except Exception as e:
                print(file_path)
                raise e
        else:
            cars_structrue_list = []
        return cars_structrue_list

    def parse_parts_info(self, response, mid):
        # 解析点击分组后的response
        res_0 = response.split("\@")[0].split("\^")
        res_1 = response.split("\@")[1].split("\^")
        mid_dict = {}  # 根据mid分类的字典
        info_dict = {}  # 以pnum为键的零件详情
        for info in res_0:
            if info == '':
                continue
            info_l = info.split("\:")
            # print(info_l)
            i0 = info_l[0]
            img_num = info_l[1]  # 描点在第几张图片上, 表示有几张图片
            i2 = info_l[2]
            pnum = info_l[3]
            imgPoint = [info_l[4], info_l[5], info_l[6], info_l[7]]
            label = info_l[8]
            k = mid + img_num + i2
            _dict = dict(
                i0=i0,
                img_num=img_num,  # 描点在第几张图片上, 表示有几张图片
                i2=i2,
                pnum=pnum,
                imgPoint=imgPoint,
                label=label
            )
            if k not in mid_dict:
                mid_dict[k] = {pnum: _dict}
            else:
                mid_dict[k][pnum] = _dict

        for info1 in res_1:
            if info1 == '':
                continue
            info1_l = info1.split("\:")
            # print(info1_l)
            info_dict[info1_l[0]] = dict(
                pnum=info1_l[0],
                label=info1_l[1],
                pid=info1_l[2],
                i3=info1_l[3],
                i4=info1_l[4],
                i5=info1_l[5],
                i6=info1_l[6],
                i7=info1_l[7],
                qty=info1_l[8],
                i9=info1_l[9],
                i10=info1_l[10],
                i11=info1_l[11],
                isImg=info1_l[12],  # 此零件是否有配件图像
                i13=info1_l[13],
                i14=info1_l[14],
                i15=info1_l[15],
                i16=info1_l[16],
            )
        # construct_mid_parts_info()
        # pprint(mid_dict)
        # pprint(info_dict)
        fina_dict = self.merge_mid_dict(mid_dict, info_dict)  # 将两个信息字典合并, 最终mid为第一层key, pnum为第二层key, 零件详情为最终value
        # pprint(fina_dict)
        return fina_dict

    def merge_mid_dict(self, mid_dict, info_dict):
        for mid, pnum_dict in mid_dict.items():
            for pnum, _dict in pnum_dict.items():
                part_info_dict = info_dict[pnum]  # 根据pnum取零件详情字典
                for k in part_info_dict:
                    _dict[k] = part_info_dict[k]
        return mid_dict

    def get_part_num_dict(self, struc_code):
        # 获取某一分组下零件列表
        url = 'http://wpc.mobis.co.kr/IllustSearch?cmd=getFinalGroupKeyList&table=CDM&group={}&match=ON&isVin=MODEL&mf=A%2C'.format(struc_code)
        self.post_data['GRNO'] = struc_code
        try:
            response, file_path = self.parse_url(url=url, method='POST', headers=self.headers, post_data=self.post_data, proxies=PROXIES)
        except:
            response, file_path = self.parse_url(url=url, method='POST', headers=self.headers, post_data=self.post_data, proxies=PROXIES)

        if response:
            part_num_dict = self.parse_parts_info(response, struc_code)
        else:
            part_num_dict = {}
        return part_num_dict

    def get_imgs(self, cid, img_list):
        # 暂时没有找到如何区分图片格式, 所以两种格式都尝试抓取

        def get_img_url(cid, img_name, format):
            t = time.time().__str__().replace('.', '')[:13]
            _dict = {
                'png': {'folder': 'PNG'},
                'gif': {'folder': 'TIF'}
            }
            base_url = 'http://wpc.mobis.co.kr/illust/GROUP/{}/{}/{}.{}?time={}'
            return base_url.format(cid, _dict[format]['folder'], img_name, format, t)

        format_l = ['png', 'gif']
        for img_name in img_list:
            cid_path = self.img_path + '{}'.format(cid)
            if not os.path.exists(cid_path):  # cid路径
                os.mkdir(cid_path)
            img_path_png = cid_path + '/{}.{}'.format(img_name, 'png')
            img_path_gif = cid_path + '/{}.{}'.format(img_name, 'gif')
            if os.path.exists(img_path_png) or os.path.exists(img_path_gif):  # 图片存在任意其一,则跳过
                continue
            else:
                for format in format_l:
                    img_url = get_img_url(cid, img_name, format)
                    try:
                        response = requests.get(url=img_url, headers=self.headers, cookies=self.cookies, proxies=PROXIES, allow_redirects=False, timeout=30)
                    except:
                        response = requests.get(url=img_url, headers=self.headers, cookies=self.cookies, proxies=PROXIES, allow_redirects=False, timeout=30)
                    status_code = response.status_code
                    if status_code == 302:  # 证明图片格式不对,尝试抓取下一种格式
                        continue
                    else:
                        fina_img_path = cid_path+'/{}.{}'.format(img_name, format)  # 最终图片路径
                        with open(fina_img_path, 'wb') as i:
                            i.write(response.content)
                        break

    def parse_part_multi(self, response):

        def func(s):
            if s and '[' in s:  # 有sa
                sa = re.findall('\[(.*?)\]', s)[0]
                uc = s.replace(sa, "").strip(" []")
                uc = uc.split(",") if uc else []
                sa = sa.strip().split(",") if sa.strip() else []
            elif s and '[' not in s:  # 只有uc
                sa = []
                uc = s.strip().split(',') if s.strip() else []
            else:
                sa = []
                uc = []
            return sa, uc


        # 解析零件multi信息
        res_l = response.split("\@")
        _l = []
        for info in res_l:
            if info.strip(" ") == "":
                continue
            info_l = info.split("\:")
            # print(info_l)
            sa, uc = func(info_l[13])
            _dict = {
                "years": info_l[0]+'-'+info_l[1],
                "pid": info_l[2],
                "label": info_l[3],
                "remark": info_l[4].replace("[", "").replace("]", ""),
                # "i5": info_l[5],
                "qty": info_l[6],
                "price": info_l[7],
                "sa": sa,
                "uc": uc
            }
            _l.append(_dict)
        return _l

    def get_part_multi(self,struc_code , pnum):
        url = 'http://wpc.mobis.co.kr/TextSearch?cmd=getFinalGroupDataList&table=CDM&group={}&pncd={}&match=ON&isVin=MODEL&mf=A%2C&testa=testa'.format(struc_code, pnum)
        self.post_data['PNCD'] = pnum
        try:
            response, file_path = self.parse_url(url=url, method='POST', headers=self.headers, post_data=self.post_data, proxies=PROXIES)
        except:
            response, file_path = self.parse_url(url=url, method='POST', headers=self.headers, post_data=self.post_data, proxies=PROXIES)
        if response:
            part_multi = self.parse_part_multi(response)
        else:
            part_multi = []
        return part_multi

    def save2db(self, table, data):
        sql_tail = ' on duplicate key update status=values(`status`)'
        in_sql = self.db.get_insert_sql(table, data) + sql_tail
        self.db.execute_sql(in_sql)

    def insert_cars_info(self, type, name, cid, years):
        county = COUNTY_DICT[self.regn]
        str_ = county+self.brands+type+name+cid+years
        _dict = {
            "uid": self.get_md5(str_.encode()),
            "name": county,
            "brand": self.brands,
            "brandCode": self.brands,
            "condition": type,
            "limit": name,
            "limit2": "",
            "limit3": "",
            "limit4": "",
            "label": "",
            "code": cid,
            "description": years,
            "imagePath": "",
            "imgName": "",
            "extra_info": "",
            "createtime": "now()",
            "updatetime": "now()",
            "status": 1,
        }
        self.save2db(INFO_TABLE, _dict)

    def insert_group_table(self, cid, num, group_name, group_code):
        str_ = self.brands+cid+str(num)+group_name+group_code
        _dict = {
            "uid": self.get_md5(str_.encode()),
            "cid": cid,
            "num": num,
            "brandCode": self.brands,
            "label": group_name,
            "extra_info": group_code,
            "updatetime": "now()",
            "createtime": "now()",
            "status": 1
        }
        self.save2db(GROUP_TABLE, _dict)

    def insert_structure_table(self, cid, num, fina_mid, structrue_name):
        str_ = cid+self.brands+str(num)+fina_mid+structrue_name
        _dict = {
            "uid": self.get_md5(str_.encode()),
            "cid": cid,
            "num": num,
            "maingroup": num,
            "subgroup": fina_mid,
            "mid": fina_mid,
            "brandCode": self.brands,
            "name": structrue_name,
            "description": "",
            "model": "",
            "condition": "",
            "imgName": "",
            "imageLarge": "",
            "imgPoints": "",
            "logo": "",
            "extra_info": "",
            "updatetime": "now()",
            "createtime": "now()",
            "status": 1,
        }
        self.save2db(STRUCTRUE_TABLE, _dict)

    def insert_part_num_table(self, cid, num, fina_mid, pnum, psort, pid, label, qty, imgPoint, years="-", remark="", sa=[], uc=[]):
        str_ = cid+self.brands+str(num)+fina_mid+pnum+str(psort)+pid+label+qty
        extra = {"date": years, "remark": remark, "sa": sa, "uc": uc, "itid": pnum}

        # 新增颜色码过滤
        if pid[-3:] in COLOR_SET:
            part_color = pid[-3:]
        elif pid[-2:] in COLOR_SET:
            part_color = pid[-2:]
        else:
            part_color = ''
        extra["part_color"] = part_color

        _dict = {
            "cid": cid,
            "uid": self.get_md5(str_.encode()),
            "num": num,
            "maingroup": num,
            "subgroup": fina_mid,
            "mid": fina_mid,
            "brandCode": self.brands,
            "pnum": pnum,
            "itid": pnum,
            "psort": psort,
            "pid": pid,
            "label": label,
            "remark": remark,
            "quantity": qty,
            "model": "",
            "isreplace": "",
            "replacelabel": "",
            "rid": "",
            "condition": "",
            "img": json.dumps(imgPoint, ensure_ascii=False),
            "logo": "",
            "price": "",
            "extra_info": json.dumps(extra, ensure_ascii=False),
            "updatetime": "now()",
            "createtime": "now()",
            "status": 1,
        }
        self.save2db(PART_NUM_TABLE, _dict)

    def run(self):
        # 先获取当前品牌下所有车系
        cars_model_list = self.get_cars_model_list()
        for cars_model in cars_model_list:
            crnm = cars_model.get("crnm")  # 车系名
            type = cars_model.get("type")  # 车的类型, 商用, 乘客, SUV, ECO
            type_code = cars_model.get("type_code")  # 车类型代号
            print(crnm)
            cars_version_list = self.get_cars_version_list(crnm, type_code)
            for cars_version in cars_version_list:
                name = cars_version.get("name")
                years = cars_version.get("years")
                cid = cars_version.get("cid")
                print(name, years, cid)

                ###TODO 在此添加检测cid代码块, 或者插入cars_info表代码块
                # if cid not in HAS_CID_LIST:
                #     print(cid, type, crnm, name)
                #     continue
                # else:
                #     continue
                # 插入cars_info_table
                self.insert_cars_info(type, name, cid, years)

                ###
                # 请求此车型必要信息, 用于以后所有请求中的post_data
                cars_detail = self.get_cars_detail(cid)
                self.reset_post_data(cars_detail)  # 重置post_data
                BPNO = cars_detail.get("BPNO")
                PCCD = cars_detail.get("PCCD")
                cars_group_list = self.get_cars_group_list(BPNO, PCCD, cid)
                num = 0  # 主组计数
                for cars_group in cars_group_list:
                    # time.sleep(1)
                    print(cars_group, "# 请求该主组")
                    group_code = cars_group.get("code")
                    group_name = cars_group.get("group_name")
                    ###TODO 在此添加插入主组信息表代码块
                    # 插入group_table
                    num += 1  # 主组计数
                    self.insert_group_table(cid, num, group_name, group_code)

                    ###
                    # 请求主组下分组信息
                    cars_structrue_list = self.get_cars_structrue_list(group_code)
                    for cars_structrue in cars_structrue_list:
                        # time.sleep(1)
                        structrue_name = cars_structrue.get("structrue_name")
                        mid = cars_structrue.get("GRNO")  # mid
                        print(structrue_name, mid, "# 请求该分组")
                        # 点击分组
                        part_num_dict = self.get_part_num_dict(mid)
                        img_list = part_num_dict.keys()
                        self.get_imgs(cid, img_list)
                        for key, value in part_num_dict.items():
                            fina_mid = key  #最终mid
                            ### TODO 在此插入structure_table
                            self.insert_structure_table(cid, num, fina_mid, structrue_name)
                            psort = 0
                            for k, v in value.items():
                                pnum = k
                                pid = v.get("pid")
                                label = v.get("label")
                                psort += 1
                                imgPoint = v.get("imgPoint")

                                if pid in ['MULTI']:  # 零件有多个, 需重新请求, 或许不仅仅是MULTI
                                    print(label, pnum, "# 请求该multi")
                                    part_num_multi = self.get_part_multi(mid, pnum)
                                    for part in part_num_multi:
                                        pid = part.get("pid")
                                        years = part.get("years")
                                        label = part.get("label")
                                        remark = part.get("remark")
                                        qty = part.get("qty")
                                        sa = part.get("sa")
                                        uc = part.get("uc")
                                        self.insert_part_num_table(cid, num, fina_mid, pnum, psort, pid, label, qty, imgPoint, years, remark, sa, uc)

                                else:
                                    ### TODO 插入零件表
                                    # remark = v.get("remark")
                                    qty = v.get("qty")
                                    self.insert_part_num_table(cid, num, fina_mid, pnum, psort, pid, label, qty, imgPoint)
        self.db.commit_sql()

if __name__ == '__main__':
    brands = 'hyundai'  # kia
    regn = 'CHN'
    tools = HK_Spider(
        db_config=DB_CONFIG,
        brands=brands,
        regn = regn
    )
    tools.run()




